package edu.hit.pact.pgse.crawler;

import java.net.InetSocketAddress;
import java.util.concurrent.LinkedBlockingQueue;

import org.apache.commons.logging.Log;

import edu.hit.pact.pgse.bean.RawWebPage;
import edu.hit.pact.pgse.bean.RunnableTask;
import edu.hit.pact.pgse.bean.TaskPieceProperty;
import edu.hit.pact.pgse.crawler.download.WebPageFetcher;
import edu.hit.pact.pgse.crawler.finish.FinishedReportedProcessor;
import edu.hit.pact.pgse.crawler.send.RawPageSendProcessor;
import edu.hit.pact.pgse.crawler.store.FileSystemWebPageStore;
import edu.hit.pact.pgse.crawler.store.WebPageStore;
import edu.hit.pact.pgse.crawler.util.CustomHttpClient;
import edu.hit.pact.pgse.util.Globals;

public abstract class Crawler {

	protected TaskPieceProperty pieceProperty = null;
	protected WebPageFetcher fetcher= null;
	protected WebPageStore store= null;
	protected FinishedReportedProcessor finishedReporter = null;
	protected InetSocketAddress storeAddr = null;
	protected LinkedBlockingQueue<RawWebPage> fetchedPageQueue = null;
	protected RawPageSendProcessor pageSendProcessor = null;
	protected Log logger = null;
	
	protected boolean ifSend;
	protected boolean ifStore;

	public Crawler(RunnableTask runnableTask, FinishedReportedProcessor finishedReporter, Log logger, boolean ifSend, boolean ifStore){
		this.logger = logger;
		this.pieceProperty = runnableTask.getTaskPieceProperty();
		this.fetcher = buildWebPageFetcher();
		this.finishedReporter = finishedReporter;
		this.ifSend = ifSend;
		this.ifStore = ifStore;
		this.fetchedPageQueue = new LinkedBlockingQueue<RawWebPage>();
		
		if (ifSend){
			this.pageSendProcessor = new RawPageSendProcessor(fetchedPageQueue, runnableTask, ifSend);
		}
		
		this.store = new FileSystemWebPageStore(ifStore);
	}
	
	public WebPageFetcher buildWebPageFetcher(){
		/*
		 * Construct HttpClient and WegPageFetcher
		 */
		logger.info("construct httpclient.");
		int maxPerHostConnection = Globals.CRAWLER_DEFAULT_MAX_PERHOST_CONNECTIONS;
		int maxTotalConnection = Globals.CRAWLER_DEFAULT_MAX_TOTAL_CONNECTIONS;
		int connectTimeout = Globals.CRAWLER_CONNECT_TIMEOUT;
		CustomHttpClient httpClient = new CustomHttpClient(maxPerHostConnection, maxTotalConnection, connectTimeout);
		
		if (Globals.CRAWLER_USE_PROXY) {
			String proxyIp = Globals.CRAWLER_PROXY_IP;
			int proxyPort = Globals.CRAWLER_PROXY_PORT;
			
			logger.info("set proxy : ip :" + proxyIp + " port :" + proxyPort);			
			httpClient.setProxy(proxyIp, proxyPort);
		}//end if
		
		logger.info("construct web page fetcher.");
		String defalutCharSet = Globals.CRAWLER_WEB_DEFAULT_CHARSET;
		int readTimeout = Globals.CRAWLER_READ_TIMEOUT;
		int maxContentLength = Globals.CRAWLER_MAX_CONTENT_LENGTH;
		
		WebPageFetcher webPageFetcher = new WebPageFetcher(httpClient, defalutCharSet, readTimeout, maxContentLength);
		return webPageFetcher;
	}
	
	public abstract void start() throws Exception;
	
	public void deactivate(){
		pieceProperty = null;
		fetcher = null;
		store = null;
//		finishedReporter.setDeadLoop(false);
		storeAddr = null;
//		fetchedPageQueue = null;
//		if (ifSend){
//			pageSendProcessor.setDeadLoop(false);
//		}
	}
}
